﻿Imports System.Text.RegularExpressions
Imports Microsoft.VisualBasic
Imports System.Net

Module WebServices

	Public Function GetPage(URL As String) As String
		Using client As New System.Net.WebClient
			Do
				'we'll keep trying this until we get a response we understand
				Try
					Dim Page As String = GetPageWorker(client, URL)
					If Page.Contains("<h2 style=""font-size:150%"">PAGE NOT FOUND</h2>") Then
						Return Nothing
					Else
						Return Page
					End If
				Catch ex As System.Net.WebException When ex.Message.Contains("404")
					Return Nothing
					'the server didn't respond
				Catch ex As System.Net.WebException When ex.Message.Contains("400")
					Return Nothing
					'the server didn't have the page we wanted
				Catch ex As System.Net.WebException When CType(ex.InnerException, System.Net.Sockets.SocketException).ErrorCode = 10060
				End Try
			Loop
		End Using
	End Function

	Public Function GetPageWorker(client As System.Net.WebClient, URL As String) As String
		If URL.Contains("homenet.org") Then
			client.Headers.Add("User-Agent", "Mozilla/5.0 (compatible; Windows NT 6;)")

			Select Case True
				Case URL.StartsWith("pinyin~")
					URL = URL.Replace("pinyin~", "")
					client.Headers.Add("Cookie", "adjustNoFilter=0") 'Pinyin
				Case URL.StartsWith("sydneylau~")
					URL = URL.Replace("sydneylau~", "")
					client.Headers.Add("Cookie", "adjustNoFilter=2") 'Sydney Lau
				Case URL.StartsWith("zhuyin~")
					URL = URL.Replace("zhuyin~", "")
					client.Headers.Add("Cookie", "adjustNoFilter=5") 'Zhuyin
			End Select
		End If
		client.Encoding = System.Text.Encoding.UTF8
		Dim Page As String = client.DownloadString(URL)
		'small pause for non-jw.org servers
		If Not URL.Contains("jw.org") Then
			Threading.Thread.Sleep(1000)
		End If
		Return Page
	End Function

	Public Function GetVersesFromPage(Page As String) As List(Of KeyValuePair(Of Integer, String))
		Dim verseCollection As New List(Of KeyValuePair(Of Integer, String))
		Dim IsPinyin As Boolean = False

		'if we run all these regex in order we will get a set of matches that looks like:
		'dcv_[chapter]_[verse]
		'verse text
		Dim removePageMarkers As New Regex("<span[^<>]*?class='pg'[^<>]*?>", RegexOptions.None)
		Dim getChapter As New Regex("<div class=('|"")document bible('|"")(.*)<div id=(""|)contentFooter(""|)>", RegexOptions.Singleline)
		Dim removeHyperlinks As New Regex("<a.*?/a>", RegexOptions.None)
		Dim removeVerseNumbers As New Regex("(?<=(dcv[_0-9]*.*?>\s*?))(<b>\s*?[0-9]+?\s*?</b>)|(<b>\s*?<sup>\s*?[0-9]+?\s*?</sup>\s*?</b>)|(<b>\s*?<sup><span class='alt'>\s*?[0-9]+?\s*?</span></sup>\s*?</b>)", RegexOptions.Singleline)
		Dim removeLineFeed As New Regex("\n", RegexOptions.Singleline)

		'special set of regex to replace the paragraph tags with line breaks when appropriate
		Dim findParagraph As New Regex("<p.*?</p>", RegexOptions.None)
		Dim paragraphReplacementEvaluator As New MatchEvaluator(AddressOf IsInVerseParagraph)

		'formatting regex
		'italics
		Dim removeItalicsRegex As String = "(?<=(<i>))(?<italics>.*?)(?=(</i>))"
		Dim replaceItalicsRegex As String = "{\i ${italics}}"
		'underline
		Dim removeUnderlineRegex As String = "(?<=(<u>))(?<underline>.*?)(?=(</u>))"
		Dim replaceUnderlineRegex As String = "{\ul ${underline}}"
		'bold
		Dim removeBoldRegex As String = "(?<=(<b>))(?<bold>[^<>]*?)(?=(</b>))"
		Dim replaceBoldRegex As String = "{\b ${bold}}"
		'superscript
		Dim removeSupRegex As String = "(?<=(<sup>))(?<sup>.*?)(?=(</sup>))"
		Dim replaceSupRegex As String = "{\super ${sup}}"

		'this will also remove any stray 'a' tags
		Dim removeTags As New Regex("(<|</)(div|p|b|i|a|ruby|rb|u|sup).*?>", RegexOptions.None)
		Dim removeClosingSpan As New Regex("</span>", RegexOptions.None)
		Dim removeExtraSpan As New Regex("<span[^_0-9]*?>", RegexOptions.None)

		'special for pinyin
		'won't hurt other pages
		IsPinyin = Page.Contains("<rt>")
		Dim removeChinesePicturePeriod As String = "<img src=""/@__3002"" width=16 height=18>"
		Dim replaceChinesePicturePeriod = "。"
		Dim removeTrailingRTTag As New Regex("</rt>", RegexOptions.Singleline)
		Dim removeRTWithLeadingSpace As New Regex("(?<= )<rt>", RegexOptions.Singleline)
		Dim replaceRTWithSpace As New Regex("<rt>", RegexOptions.Singleline)
		Dim removeRB As New Regex("<rb>.*?</rb>", RegexOptions.Singleline)

		Dim matchVerseNumberThenText As New Regex("(dcv[_0-9]*)|((?<=>).*?(?=(<|$)))", RegexOptions.Singleline)

		Page = removePageMarkers.Replace(Page, "")
		Page = getChapter.Match(Page).Value
		Page = removeLineFeed.Replace(Page, "")

		Page = Regex.Replace(Page, removeItalicsRegex, replaceItalicsRegex)
		If Not IsPinyin Then
			'the underlining would come from Chinese, but is unneeded in pinyin and doesn't turn out very well
			Page = Regex.Replace(Page, removeUnderlineRegex, replaceUnderlineRegex)
		End If
		Page = Regex.Replace(Page, removeChinesePicturePeriod, replaceChinesePicturePeriod)

		Page = removeHyperlinks.Replace(Page, "")
		Page = removeVerseNumbers.Replace(Page, "")

		'these must come after removing the verses because verses are bolded and superscripted
		Page = Regex.Replace(Page, removeSupRegex, replaceSupRegex)
		Page = Regex.Replace(Page, removeBoldRegex, replaceBoldRegex)

		'If Page.Contains("<b>") Then
		'	Stop
		'End If

		Page = findParagraph.Replace(Page, paragraphReplacementEvaluator)
		'fix for duplicated line breaks
		Page = Page.Replace("{\line }{\line }", "{\line }")

		'must come before we remove all the RB tags
		Page = removeRB.Replace(Page, "")

		Page = removeTags.Replace(Page, "")
		Page = removeClosingSpan.Replace(Page, "")
		Page = removeExtraSpan.Replace(Page, "")
		Page = removeTrailingRTTag.Replace(Page, "")
		Page = removeRTWithLeadingSpace.Replace(Page, "")
		Page = replaceRTWithSpace.Replace(Page, " ")

		'only for pinyin, fix quote spacing
		If IsPinyin Then
			Dim fixLeftDoubleQuote As New Regex("(" & ChrW(&H201C) & " )|(?<! )" & ChrW(&H201C), RegexOptions.Singleline)
			Dim fixRightDoubleQuote As New Regex("( " & ChrW(&H201D) & ")|" & ChrW(&H201D) & "(?!\W)", RegexOptions.Singleline)
			Dim fixLeftSingleQuote As New Regex("(" & ChrW(&H2018) & " )|(?<! )" & ChrW(&H2018), RegexOptions.Singleline)
			Dim fixRightSingleQuote As New Regex("( " & ChrW(&H2019) & ")|" & ChrW(&H2019) & "(?!\W)", RegexOptions.Singleline)

			Page = fixLeftDoubleQuote.Replace(Page, " " & ChrW(&H201C))
			Page = fixRightDoubleQuote.Replace(Page, ChrW(&H201D) & " ")
			Page = fixLeftSingleQuote.Replace(Page, " " & ChrW(&H2018))
			Page = fixRightSingleQuote.Replace(Page, ChrW(&H2019) & " ")
		End If

		Dim pageMatches As MatchCollection = matchVerseNumberThenText.Matches(Page)

		For i = 0 To pageMatches.Count - 1
			Dim verseNumber As Integer = pageMatches(i).Value.Split("_").Last
			i += 1
			Dim verseText As String = pageMatches(i).Value
			Dim ASCIIText As String = verseText.UnicodeToASCII
			verseCollection.Add(New KeyValuePair(Of Integer, String)(verseNumber, verseText.Trim))
		Next

		Return verseCollection

	End Function

	Public Function IsInVerseParagraph(myMatch As Match) As String
		Dim countVerses As New Regex("dcv[_0-9]*", RegexOptions.None)
		Dim findOpeningParagraphTax As New Regex("<p.*?>", RegexOptions.None)
		Dim findClosingParagraphTag As New Regex("</p>", RegexOptions.None)

		Dim paragraph As String = myMatch.Value
		Select Case countVerses.Matches(paragraph).Count
			Case 0
				'	Return paragraph.Replace("</p>", Environment.NewLine)
				'Case 1
				Return findOpeningParagraphTax.Replace(paragraph, "{\line }").Replace("</p>", "{\line }")
			Case Else
				Return paragraph
		End Select

	End Function

	Public Function GetNextURL(BookNumber As Integer, Chapter As Integer, BibleBaseURL As String) As String
		Dim URL As String = String.Format("{0}{1}/{2}", BibleBaseURL, BookNumber, Chapter)

		Return URL

	End Function

End Module
